{
 "cells": [
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-26T12:08:28.372564Z",
     "start_time": "2025-01-26T12:08:26.436708Z"
    }
   },
   "source": [
    "import torch"
   ],
   "outputs": [],
   "execution_count": 1
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# torch 求导\n",
    "\n",
    "参考 [url](https://pytorch.org/tutorials/beginner/basics/autogradqs_tutorial.html)\n",
    "\n",
    "pytorch 实现模型训练需要完整地写下训练过程，包括反向传播求梯度以及应用梯度下降算法。（06见chapter_2/03_...)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 近似求导"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-26T12:09:54.366689Z",
     "start_time": "2025-01-26T12:09:54.363244Z"
    }
   },
   "source": [
    "def f(x):\n",
    "    return 3. * x ** 2 + 2. * x - 1\n",
    "#近视求导，x移动eps单位，也就是离自己很近的一个点的切线\n",
    "def approximate_derivative(f, x, eps=1e-6):\n",
    "    return (f(x + eps) - f(x - eps)) / (2. * eps)\n",
    "\n",
    "print(approximate_derivative(f, 1.)) # 6x+2=8"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7.999999999785956\n"
     ]
    }
   ],
   "execution_count": 3
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-26T12:12:37.007410Z",
     "start_time": "2025-01-26T12:12:37.003390Z"
    }
   },
   "source": [
    "# 求偏导数,其中一个数不动，对另外一个变量求导\n",
    "def g(x1, x2):\n",
    "    return (x1 + 5) * (x2 ** 2)\n",
    "\n",
    "def approximate_gradient(g, x1, x2, eps=1e-3):\n",
    "    dg_x1 = approximate_derivative(lambda x: g(x, x2), x1, eps) # x2对x是常数\n",
    "    dg_x2 = approximate_derivative(lambda x: g(x1, x), x2, eps)\n",
    "    return dg_x1, dg_x2\n",
    "\n",
    "print(approximate_gradient(g, 2., 3.))"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(8.999999999993236, 41.999999999994486)\n"
     ]
    }
   ],
   "execution_count": 5
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## torch 近似求导"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-26T12:16:21.119272Z",
     "start_time": "2025-01-26T12:16:21.114574Z"
    }
   },
   "source": [
    "# 声明两个tensor x1 和 x2，允许梯度计算，使用torch的自动求导上下文计算两个tensor的梯度\n",
    "# 使用 torch.autograd.grad 计算 y = g(x1, x2) 的偏导数\n",
    "\n",
    "x1 = torch.tensor([2.], requires_grad=True)\n",
    "x2 = torch.tensor([3.], requires_grad=True)\n",
    "y = g(x1, x2)\n",
    "    \n",
    "(dy_dx1,) = torch.autograd.grad(y, x1,retain_graph=True)  # True表示保留计算图\n",
    "print(dy_dx1)"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([9.])\n"
     ]
    }
   ],
   "execution_count": 24
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-26T12:16:21.776470Z",
     "start_time": "2025-01-26T12:16:21.772372Z"
    }
   },
   "cell_type": "code",
   "source": [
    "try:\n",
    "    (dy_dx2,) = torch.autograd.grad(y, x2)  # 未保存计算图的话，第二次执行报错\n",
    "    print(dy_dx2)\n",
    "except Exception as e:\n",
    "    print(e)"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([42.])\n"
     ]
    }
   ],
   "execution_count": 25
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-26T12:16:23.741766Z",
     "start_time": "2025-01-26T12:16:23.737430Z"
    }
   },
   "source": [
    "# 同时求导\n",
    "x1 = torch.tensor([2.], requires_grad=True)\n",
    "x2 = torch.tensor([3.], requires_grad=True)\n",
    "y = g(x1, x2)\n",
    "\n",
    "# 求偏导数\n",
    "dy_dx1, dy_dx2 = torch.autograd.grad(y, [x1, x2])\n",
    "print(dy_dx1, dy_dx2)"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([9.]) tensor([42.])\n"
     ]
    }
   ],
   "execution_count": 26
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-26T12:17:07.108506Z",
     "start_time": "2025-01-26T12:17:07.103462Z"
    }
   },
   "source": [
    "# 当然我们一般直接用 backward\n",
    "\n",
    "x1 = torch.tensor([2.], requires_grad=True)\n",
    "x2 = torch.tensor([3.], requires_grad=True)\n",
    "y = g(x1, x2)\n",
    "\n",
    "# 求偏导数,求梯度;.grab存梯度\n",
    "y.backward()\n",
    "print(x1.grad, x2.grad)"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([9.]) tensor([42.])\n"
     ]
    }
   ],
   "execution_count": 27
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 二阶导\n"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-26T12:18:16.764559Z",
     "start_time": "2025-01-26T12:18:16.759545Z"
    }
   },
   "source": [
    "x1 = torch.tensor([2.], requires_grad=True)\n",
    "x2 = torch.tensor([3.], requires_grad=True)\n",
    "y = g(x1, x2)\n",
    "\n",
    "# 求y对x1和x2的二阶偏导数\n",
    "#，allow_unused 参数的作用是控制当 inputs 中的某些张量不需要梯度时，函数的行为方式。\n",
    "dy_dx1, dy_dx2 = torch.autograd.grad(y, [x1, x2], create_graph=True)  # create_graph=True允许计算高阶导数\n",
    "dy_dx1_dx1, dy_dx1_dx2 = torch.autograd.grad(dy_dx1, [x1, x2], allow_unused=True)\n",
    "dy_dx2_dx1, dy_dx2_dx2 = torch.autograd.grad(dy_dx2, [x1, x2], allow_unused=True)\n",
    "print(dy_dx1_dx1, dy_dx2_dx1, dy_dx2_dx2)"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "None tensor([6.]) tensor([14.])\n"
     ]
    }
   ],
   "execution_count": 29
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-26T12:22:41.044513Z",
     "start_time": "2025-01-26T12:22:41.019114Z"
    }
   },
   "source": [
    "#模拟梯度下降算法 SGD\n",
    "import torch\n",
    "learning_rate = 0.3\n",
    "x = torch.tensor(2.0, requires_grad=True)\n",
    "for _ in range(100):\n",
    "    z = f(x)\n",
    "    z.backward()\n",
    "    x.data.sub_(learning_rate * x.grad) # x -= learning_rate * x.grad，这里就等价于optimizer.step()\n",
    "    x.grad.zero_() # x.grad -= x.grad, x.grad = 0,梯度清零\n",
    "print(x)"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(-0.3333, requires_grad=True)\n"
     ]
    }
   ],
   "execution_count": 30
  },
  {
   "cell_type": "code",
   "source": [
    "a=torch.tensor(2) # 标量\n",
    "a.shape"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2025-01-26T12:22:57.472872Z",
     "start_time": "2025-01-26T12:22:57.467839Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([])"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 31
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-26T12:24:46.235852Z",
     "start_time": "2025-01-26T12:24:45.128307Z"
    }
   },
   "source": [
    "#GradientTape与optimizer（优化器）结合使用\n",
    "learning_rate = 0.01\n",
    "x = torch.tensor(2.0, requires_grad=True)  #x为梯度变量\n",
    "optimizer = torch.optim.SGD([x], lr=learning_rate,momentum=0.9)\n",
    "for _ in range(500):\n",
    "    z = f(x)\n",
    "    optimizer.zero_grad() # 梯度变为0\n",
    "    z.backward() # dz/dx,求梯度\n",
    "    # print(x.grad)\n",
    "    optimizer.step() # x -= learning_rate * x.grad\n",
    "\n",
    "print(x)"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(-0.3333, requires_grad=True)\n"
     ]
    }
   ],
   "execution_count": 32
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": ""
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
